from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
# import gc
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
# import numpy as np
from os import walk
# import matplotlib.pyplot as plt
from subprocess import call
import os
import collections

print "Begins..."
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = 113048112924
inputFileHandler0.seek(numOfBytesRead)
while numOfBytesRead < fileSize:
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    print docID,termID,postingHitFreq
    # exit(1)
    if docID > 10000002:
        exit(1)
    numOfBytesRead += 12
print "Ends."
exit(1)

'''
beginningPosition = 199530487296
targetDocID = 25000000
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = beginningPosition
inputFileHandler0.seek(numOfBytesRead)
while numOfBytesRead < fileSize:
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    print docID,termID,postingHitFreq
    #if docID > targetDocID:
        # decrease
    while (targetDocID - docID) > 1000:
        numOfBytesRead += 1000000 * 12
        inputFileHandler0.seek(numOfBytesRead)
        if docID < targetDocID:
            byteString = inputFileHandler0.read(4 + 4 + 4)
            (docID,termID,postingHitFreq) = unpack( "3I", byteString)
            print docID,termID,postingHitFreq,numOfBytesRead        
        if docID > targetDocID:
            print docID,targetDocID,"break"
            exit(1)
    print docID,numOfBytesRead
    numOfBytesRead += 12
print "Ends."
exit(1)
'''

# some numbers:
#        0            0
#  1000000  20407812012 
#  2000000  37979581056  
#  3000000  53446024716
#  4000000  65582188956
#  5000000  75814235460
#  6000000  85833072684
#  7000000  94580553876
#  8000000  102778986396
#  9000001  108537313656
#  10000001 113048112924
# 11000000 117472283112
# 12000002 121894212264
# 13000000 126261377304
# 14000003 130280992908
# 15000001 133414035600
# 16000000 136566045192
# 17000000 139463385108
# 18000002 142330945980
# 19000005 145112762640
# 20000001 147622084956
# 21000002 150042023376
# 22000000 165315614076
# 23000000 187715366112
# 24000000 207087755040
# 25000000 232181415996

print "Begins..."
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = 232159036008
inputFileHandler0.seek(numOfBytesRead)
while numOfBytesRead < fileSize:
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    print docID,termID,postingHitFreq
    # exit(1)
    if docID >= 25000000:
        print docID,numOfBytesRead
        exit(1)
    numOfBytesRead += 12
print "Ends."
exit(1)

print "Begins..."
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
print "currentInputFileName: ",ifn
print "file size:",fileSize
numOfBytesRead = 85513065984
inputFileHandler0.seek(numOfBytesRead)
while numOfBytesRead < fileSize:
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    print docID,termID,postingHitFreq
    if docID >= 6000000:
        print docID,numOfBytesRead
        exit(1)
    numOfBytesRead += 12
print "Ends."

print "Begins..."
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_50M/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
inputFileHandler0 = open(ifn,"rb")
statinfo = os.stat(ifn)
fileSize = statinfo.st_size
theGap = 9501451776
print "currentInputFileName: ",ifn
print "file size:",fileSize
print "theGap:",theGap
for i in range(0,25):
    inputFileHandler0.seek(i * theGap)
    byteString = inputFileHandler0.read(4 + 4 + 4)
    (docID,termID,postingHitFreq) = unpack( "3I", byteString)
    print i,i*theGap,docID,termID,postingHitFreq
    
    '''
    inputFileHandler0.seek(0)
    numOfBytesRead = 0
    numOfDocumentsProcessed = 0
    while numOfBytesRead < fileSize:
        byteString = inputFileHandler0.read(4 + 4 + 4)
        (docIDFromFile,numOfPostings,docHit) = unpack( "3I", byteString)
        print "docID:",docIDFromFile,numOfPostings,docHit
        numOfBytesRead += 12
        
        for i in range(0,numOfPostings):
            byteString = inputFileHandler0.read(4 + 4 + 4)
            (termID,score,postingHit) = unpack( "1I1f1I", byteString)
            # print "----->",i,termID,score,postingHit
            numOfBytesRead += 12
        
    inputFileHandler0.close()
    '''
    
print "Ends."




